﻿import chardet
import urllib
import urllib2
import urlparse

def safe_replace_end_line_char(s):
	ret = ''
	pos = 0
	while pos != -1:
		d = s.find('\n', pos)
		if d == 0 or (d>0 and d != s.find('\r\n', pos) - 1):
			s = s[:d] +  '\r\n' + s[d+1:]
			pos = d + 2
		elif d > 0:
			pos = d+1
		else:
			pos = -1
	return s	

def to_utf8(string):
	ss = string
	try:
		ss = string.decode('UTF-8').encode('UTF-8')
	except:
		try:
			ss = string.decode('GB18030').encode('UTF-8')
		except:
			try:
				encoding_info = chardet.detect(string)
				charset = encoding_info['encoding']
				if charset.lower() == 'ascii':
					ss = string.encode('UTF-8')
				else:
					ss = string.decode(charset).encode('UTF-8')
			except:
				return ss
	return ss

def download_page(url):
	downloader = urllib2.urlopen(url)
	ss = downloader.read()
	return to_utf8(ss)

def safe_download_page(url, retry_times=3, timeout=120):
	print 'begin download : ' + url
	counter = 0
	while counter < retry_times:
		try:
			headers = { 'User-Agent' : 'Mozilla/5.0' }
			req = urllib2.Request(url, None, headers)
			html = urllib2.urlopen(req).read()
			downloader = urllib2.urlopen(req, None, timeout)
			content = downloader.read()
			return content
		except:
			print 'download error...'
			counter += 1

def get_tid(url):
#	url = "read.php?tid=1596332&fpage=2"
	result = urlparse.urlparse(url)
#	assert(result.path == '/viewthread.php' or result.path == 'viewthread.php')
	query = urlparse.parse_qs(result.query, True)
	return query['tid'][0]
